Archived: Python Craigslist Search Script

calatron · 05-02-2010, 09:38 PM

http://cal.freeshell.org/2010/04/pyt...search-script/

It's a pretty simple script. It performs a search for your query on any number of areas.

Start the script and enter the cities you want separated by commas but no spaces:
newyork,albany,buffalo

Then enter your query as normal:
dell laptop

It will create an html document with all the results for that query in those areas.

Download: http://cal.freeshell.org/wp-content/...10/04/clget.py

Code:

Code:

import re
import datetime
import time
import string
import urllib
import urllib2
 
results = re.compile('<p>.+<div>sort by')
delay = 100
 
t = datetime.datetime.now()
tyme = time.mktime(t.timetuple())
 
print "Welcome to CLget!"
print ""
 
cityIN = raw_input('City or cities separated by commas and no spaces: ')
query = raw_input("Input query: ")
 
s = ','
 
cities = re.split(s,cityIN)
 
for city in list(set(cities)):
    url = "http://" + city + ".craigslist.org/search/?areaID=126&subAreaID=&query=" + query.replace(' ', ',') + "&catAbb=sss"
 
    #Setup headers to spoof Mozilla
    dat = None
    ua = "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.9.1.4) Gecko/20091007 Firefox/3.5.4"
    head = {'User-agent': ua}
 
    errorcount = 0
 
    #Get page
    req = urllib2.Request(url, dat, head)
    try:
        response = urllib2.urlopen(req)
    except urllib2.HTTPError, e:
        if errorcount < 1:
            errorcount = 1
            print "Request failed, retrying in " + delay + " seconds"
            time.sleep(int(delay))
            response = urllib2.urlopen(req)
    except urllib2.URLError, e:
        if errorcount < 1:
            errorcount = 1
            print "Request failed, retrying in " + delay + " seconds"
            time.sleep(int(delay))
            response = urllib2.urlopen(req)
 
    msg = response.read()
    errorcount = 0
 
    res = results.findall(msg)
    res = str(res)
    res = res.replace('[', '')
    res = res.replace(']', '')
    res = res.replace('sort by', '')
    res = res.replace(chr(39), '')
 
    outp = open("results" + str(tyme) + ".html", "a")
    outp.write(city)
    outp.write(str(res))
    outp.close()